from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

NUM_QUERY_TERM_POSITIONS = 351734

# step1:
# key: freqKey
# value: probability
freqKeyWithCorrespondingProbability = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_good_turing_output"
inputFileHandler = open(inputFileName,"r")

inputFileHandler.readline()
inputFileHandler.readline()
inputFileHandler.readline()
inputFileHandler.readline()

for line in inputFileHandler.readlines()[0:20]:
    lineElements = line.strip().split(" ")
    freqKey = lineElements[0]
    probability = lineElements[3]
    if freqKey not in freqKeyWithCorrespondingProbability:
        freqKeyWithCorrespondingProbability[freqKey] = probability
    else:
        print "mark1"
        exit(1)

print "freqKeyWithCorrespondingProbability:",freqKeyWithCorrespondingProbability
print "len(freqKeyWithCorrespondingProbability):",len(freqKeyWithCorrespondingProbability)
print "freqKeyWithCorrespondingProbability['1']:",freqKeyWithCorrespondingProbability['1']
inputFileHandler.close()

# step2:
# key: query term
# value: freq of freq class they belong to
inverseQueryTermFreqDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_with_query_terms_sortedByFreqR"
inputFileHandler = open(inputFileName,"r")

for line in inputFileHandler.readlines()[0:19]:
    lineElements = line.strip().split(" ")
    freq = lineElements[0]
    numOfQueryTemrs = int(lineElements[1])
    
    for i in range(2,2+numOfQueryTemrs):
        # print lineElements[i]
        
        if lineElements[i] not in inverseQueryTermFreqDict:
            inverseQueryTermFreqDict[ lineElements[i] ] = freq
        else:
            print "Unexpected Behaviour"
            exit(1)
        
print "len(inverseQueryTermFreqDict):",len(inverseQueryTermFreqDict)
# 248 2 mental time

# print "inverseQueryTermFreqDict:",inverseQueryTermFreqDict
inputFileHandler.close()

# step3:
# aux step begins...
queryTermsWithRealFreqIn85KDict = {}
inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%"
inputAuxFileHandler = open(inputAuxFileName,"r")
for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    realFreqIn85KQueries = int(lineElements[1])
    if queryTerm not in queryTermsWithRealFreqIn85KDict:
        queryTermsWithRealFreqIn85KDict[queryTerm] = realFreqIn85KQueries
    else:
        print "Unexpected Behaviour, Mark3"
        exit(1)

print "len(queryTermsWithRealFreqIn85KDict):",len(queryTermsWithRealFreqIn85KDict)
inputAuxFileHandler.close()
# aux step ends.
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom100KWithTheirTrueProbablityAndOurOwnModelPredictedProbablity1D_2D"
inputFileHandler = open(inputFileName,"r")

outputFileName = inputFileName + "_GoodTuringProbabilityAdded20130429"
outputFileHandler = open(outputFileName,"w")
 
oldHeadLine = inputFileHandler.readline()
newHeadLine = oldHeadLine.strip() + " " + "goodTuring" + "\n"
outputFileHandler.write(newHeadLine)

numOfSeenTerms = 0
numOfUnseenTerms = 0

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    if queryTerm in inverseQueryTermFreqDict:
        currentQueryTermProbability = freqKeyWithCorrespondingProbability[ inverseQueryTermFreqDict[queryTerm] ]
        outputFileHandler.write( line.strip() + " " + currentQueryTermProbability + "\n")
        numOfSeenTerms += 1
    else:
        # The currentQueryTermProbability will be the unseen query term probability: freqKeyWithCorrespondingProbability[ "0" ]
        if queryTerm in queryTermsWithRealFreqIn85KDict and queryTermsWithRealFreqIn85KDict[queryTerm] >= 20:
            currentQueryTermProbability = queryTermsWithRealFreqIn85KDict[queryTerm] / NUM_QUERY_TERM_POSITIONS
            if queryTerm == "of":
                print "mark1",currentQueryTermProbability
            numOfSeenTerms += 1
        else:
            currentQueryTermProbability = freqKeyWithCorrespondingProbability[ "0" ]
            if queryTerm == "of":
                print "mark2",currentQueryTermProbability
            numOfUnseenTerms += 1
        outputFileHandler.write( line.strip() + " " + str(currentQueryTermProbability) + "\n")
        

print "numOfSeenTerms:",numOfSeenTerms
print "numOfUnseenTerms:",numOfUnseenTerms
inputFileHandler.close()
outputFileHandler.close()
